#! /usr/bin/env python
# -*- coding: utf-8 -*-
#  Create by Albert_Chen
#  CopyRight (py) 2016年 陈超. All rights reserved by Chao.Chen.

__author__ = 'Albert'


import random

class RandomUserAgent(object):
    """Randomly rotate user agents based on a list of predefined ones"""

    def __init__(self, agents):
        self.agents = agents

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler.settings.getlist('USER_AGENTS'))

    def process_request(self, request, spider):
        request.headers.setdefault('User-Agent', random.choice(self.agents))